{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-06T08:09:08.393480",
     "start_time": "2017-07-06T08:09:03.000172Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt, animation\n",
    "\n",
    "from time import sleep\n",
    "\n",
    "%matplotlib notebook\n",
    "\n",
    "sns.set_context(\"paper\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-20T09:44:19.389748",
     "start_time": "2017-07-20T09:44:19.332744Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-20T08:27:15.177174",
     "start_time": "2017-07-20T08:27:14.794152Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with tf.device('/gpu:0'):\n",
    "  a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a')\n",
    "  b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b')\n",
    "c = tf.matmul(a, b)\n",
    "# Creates a session with log_device_placement set to True.\n",
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
    "# Runs the op.\n",
    "print(sess.run(c))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Unit Models\n",
    "Exploration of model for basic constituent units of a neural network."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Perceptron\n",
    "Starting with the perceptron, an early version model based on binary inputs and step function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T10:47:39.919292",
     "start_time": "2017-06-23T10:47:39.914292Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# weights\n",
    "W = np.array([-2,-2])\n",
    "# bias\n",
    "b = 3\n",
    "# threshold. Can be discarded using the bias instead (bias=-threshold)\n",
    "#threshold = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# perceptron firing rule\n",
    "perceptron = lambda x : 1 if np.dot(X, W) + b >0 else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# input array\n",
    "X = np.array([1,1])\n",
    "# compute perceptron output\n",
    "perceptron(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Sigmoid Neuron\n",
    "Allowing real input values (range 0-1) and using non linear activation function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T10:47:42.431436",
     "start_time": "2017-06-23T10:47:42.424435Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# neuron firing rule\n",
    "neuron = lambda x : 1/(1 + np.exp(-np.dot(X, W) - b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T10:47:42.978467",
     "start_time": "2017-06-23T10:47:42.967466Z"
    },
    "collapsed": false,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# input array\n",
    "X = np.array([1,1])\n",
    "# compute neuron output\n",
    "neuron(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network for Linear Regression\n",
    "Solve a line-fitting problem using a vanilla keras neural network "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-06T08:09:10.567605",
     "start_time": "2017-07-06T08:09:10.562605Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# line function\n",
    "def line(intercept, slope, x):\n",
    "    return x*slope + intercept\n",
    "\n",
    "# sin modulated line function\n",
    "def sin_line(x):\n",
    "    return np.sin(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-06T08:09:14.632837",
     "start_time": "2017-07-06T08:09:14.627837Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create our random data (line)\n",
    "n = 1000\n",
    "slope = 1.5\n",
    "intercept = 5.\n",
    "x = np.random.random(n)\n",
    "y = line(intercept, slope, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-06T08:10:01.649527",
     "start_time": "2017-07-06T08:10:01.643526Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create our random data (sin line)\n",
    "n = 1000\n",
    "x_data = np.linspace(-10., 10., n)\n",
    "y_data = sin_line(x_data) + np.random.uniform(-0.5, 0.5, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-06T08:10:03.900655",
     "start_time": "2017-07-06T08:10:03.663642Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# plot data\n",
    "sns.regplot(x_data, y_data)\n",
    "sns.plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train with TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-06T08:10:17.258419",
     "start_time": "2017-07-06T08:10:14.767277Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-05T11:51:00.914074",
     "start_time": "2017-07-05T11:51:00.872072Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Network parameters\n",
    "X = tf.placeholder(tf.float32, name='X')\n",
    "y = tf.placeholder(tf.float32, name='Y')\n",
    "W = tf.Variable(tf.random_normal([1], dtype=tf.float32, stddev=0.1), name='weight')\n",
    "b = tf.Variable(tf.constant([0], dtype=tf.float32), name='bias')\n",
    "\n",
    "# computation\n",
    "y_pred = W*X+b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-05T11:51:01.770123",
     "start_time": "2017-07-05T11:51:01.746122Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cost definition\n",
    "def cost_fun(y, y_pred):\n",
    "    return tf.abs(y-y_pred)\n",
    "\n",
    "cost = tf.reduce_mean(cost_fun(y, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-05T11:51:02.385159",
     "start_time": "2017-07-05T11:51:02.229150Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-05T15:07:14.674927",
     "start_time": "2017-07-05T15:07:09.732645Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_iters = 10000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    for i in range(n_iters):\n",
    "        sess.run(optimizer, feed_dict={X: x_data, y: y_data})\n",
    "        training_cost = sess.run(cost, feed_dict={X: x_data, y: y_data})\n",
    "\n",
    "        if i%100 == 0:\n",
    "            print(training_cost)\n",
    "    \n",
    "    ys_pred = y_pred.eval(feed_dict={X: x_data}, session=sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-05T15:07:46.647756",
     "start_time": "2017-07-05T15:07:46.559751Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1)\n",
    "sns.regplot(x_data, y_data, fit_reg=False, ax=ax)\n",
    "sns.regplot(x_data, ys_pred, fit_reg=False, ax=ax)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Train with Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-05T09:01:34.668598",
     "start_time": "2017-07-05T09:01:28.668255Z"
    },
    "collapsed": false,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from keras import models\n",
    "from keras import layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-05T09:01:34.730602",
     "start_time": "2017-07-05T09:01:34.670598Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Create neural network\n",
    "nn = models.Sequential()\n",
    "nn.add(layers.Dense(1, input_dim=1))\n",
    "nn.compile(optimizer='sgd', loss='mse')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-05T09:01:54.512733",
     "start_time": "2017-07-05T09:01:35.309635Z"
    },
    "collapsed": false,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# train model\n",
    "# dummy way of training, for the sake of retrieving our weights at each single training step\n",
    "theta_history = [] #weights\n",
    "loss_history = []\n",
    "for i in range(1000):\n",
    "    loss_history.append(nn.fit(x, y, nb_epoch=1, verbose=0).history['loss'][0])\n",
    "    theta_history.append((nn.layers[0].get_weights()[0][0][0], nn.layers[0].get_weights()[1][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-05T09:01:57.457902",
     "start_time": "2017-07-05T09:01:56.993875Z"
    },
    "collapsed": false,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Plot SGD animation\n",
    "from matplotlib import pyplot as plt, animation\n",
    "fig = sns.plt.figure(dpi=100, figsize=(5, 4))\n",
    "# original data\n",
    "sns.regplot(x, y, fit_reg=False)\n",
    "# initial parameters\n",
    "init_slope, init_intercept = theta_history[0]\n",
    "line, = plt.plot([0, 1.0], [init_intercept, line(init_intercept, init_slope, 1.0)], 'k-')\n",
    "epoch_text = sns.plt.text(0, 0, \"Epoch 0\")\n",
    "sns.plt.show()\n",
    "\n",
    "def animate(i):\n",
    "    current_slope, current_intercept = theta_history[i]\n",
    "    line.set_ydata([current_intercept, line(current_intercept, current_slope, 1.0)])\n",
    "    epoch_text.set_text(\"Epoch {}, cost {:.3f}\".format(i, loss_history[i]))\n",
    "    return line,\n",
    "\n",
    "ani = animation.FuncAnimation(fig, animate, np.arange(0, len(theta_history)), interval=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:neural-networks]",
   "language": "python",
   "name": "conda-env-neural-networks-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "86px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
